{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys, os, _pickle as pickle\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import nltk\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_dir = '../data'\n",
    "ckpt_dir = '../checkpoint'\n",
    "word_embd_dir = '../checkpoint/word_embd'\n",
    "model_dir = '../checkpoint/modelv5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_embd_dim = 100\n",
    "pos_embd_dim = 25\n",
    "dep_embd_dim = 25\n",
    "word_vocab_size = 400001\n",
    "pos_vocab_size = 10\n",
    "dep_vocab_size = 21\n",
    "relation_classes = 19\n",
    "word_state_size = 100\n",
    "other_state_size = 50\n",
    "batch_size = 10\n",
    "channels = 3\n",
    "lambda_l2 = 0.00001\n",
    "max_len_path = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"input\"):\n",
    "    path_length = tf.placeholder(tf.int32, shape=[2, batch_size], name=\"path1_length\")\n",
    "    word_ids = tf.placeholder(tf.int32, shape=[2, batch_size, max_len_path], name=\"word_ids\")\n",
    "    pos_ids = tf.placeholder(tf.int32, [2, batch_size, max_len_path], name=\"pos_ids\")\n",
    "    dep_ids = tf.placeholder(tf.int32, [2, batch_size, max_len_path], name=\"dep_ids\")\n",
    "    y = tf.placeholder(tf.int32, [batch_size], name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"word_embedding\"):\n",
    "    W = tf.Variable(tf.constant(0.0, shape=[word_vocab_size, word_embd_dim]), name=\"W\")\n",
    "    embedding_placeholder = tf.placeholder(tf.float32,[word_vocab_size, word_embd_dim])\n",
    "    embedding_init = W.assign(embedding_placeholder)\n",
    "    embedded_word = tf.nn.embedding_lookup(W, word_ids)\n",
    "    word_embedding_saver = tf.train.Saver({\"word_embedding/W\": W})\n",
    "\n",
    "with tf.name_scope(\"pos_embedding\"):\n",
    "    W = tf.Variable(tf.random_uniform([pos_vocab_size, pos_embd_dim]), name=\"W\")\n",
    "    embedded_pos = tf.nn.embedding_lookup(W, pos_ids)\n",
    "    pos_embedding_saver = tf.train.Saver({\"pos_embedding/W\": W})\n",
    "\n",
    "with tf.name_scope(\"dep_embedding\"):\n",
    "    W = tf.Variable(tf.random_uniform([dep_vocab_size, dep_embd_dim]), name=\"W\")\n",
    "    embedded_dep = tf.nn.embedding_lookup(W, dep_ids)\n",
    "    dep_embedding_saver = tf.train.Saver({\"dep_embedding/W\": W})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"word_dropout\"):\n",
    "    embedded_word_drop = tf.nn.dropout(embedded_word, 0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_hidden_state = tf.zeros([batch_size, word_state_size], name='word_hidden_state')\n",
    "word_cell_state = tf.zeros([batch_size, word_state_size], name='word_cell_state')\n",
    "word_init_state = tf.contrib.rnn.LSTMStateTuple(word_hidden_state, word_cell_state)\n",
    "\n",
    "other_hidden_states = tf.zeros([channels-1, batch_size, other_state_size], name=\"hidden_state\")\n",
    "other_cell_states = tf.zeros([channels-1, batch_size, other_state_size], name=\"cell_state\")\n",
    "\n",
    "other_init_states = [tf.contrib.rnn.LSTMStateTuple(other_hidden_states[i], other_cell_states[i]) for i in range(channels-1)]\n",
    "\n",
    "with tf.variable_scope(\"word_lstm1\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(word_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_word_drop[0], sequence_length=path_length[0], initial_state=word_init_state)\n",
    "    state_series_word1 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"word_lstm2\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(word_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_word_drop[1], sequence_length=path_length[1], initial_state=word_init_state)\n",
    "    state_series_word2 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"pos_lstm1\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(other_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_pos[0], sequence_length=path_length[0],initial_state=other_init_states[0])\n",
    "    state_series_pos1 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"pos_lstm2\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(other_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_pos[1], sequence_length=path_length[1],initial_state=other_init_states[0])\n",
    "    state_series_pos2 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"dep_lstm1\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(other_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_dep[0], sequence_length=path_length[0], initial_state=other_init_states[1])\n",
    "    state_series_dep1 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"dep_lstm2\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(other_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_dep[1], sequence_length=path_length[1], initial_state=other_init_states[1])\n",
    "    state_series_dep2 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "state_series1 = tf.concat([state_series_word1, state_series_pos1, state_series_dep1], 1)\n",
    "state_series2 = tf.concat([state_series_word2, state_series_pos2, state_series_dep2], 1)\n",
    "\n",
    "state_series = tf.concat([state_series1, state_series2], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"hidden_layer\"):\n",
    "    W = tf.Variable(tf.truncated_normal([400, 100], -0.1, 0.1), name=\"W\")\n",
    "    b = tf.Variable(tf.zeros([100]), name=\"b\")\n",
    "    y_hidden_layer = tf.matmul(state_series, W) + b\n",
    "\n",
    "with tf.name_scope(\"dropout\"):\n",
    "    y_hidden_layer_drop = tf.nn.dropout(y_hidden_layer, 0.3)\n",
    "\n",
    "with tf.name_scope(\"softmax_layer\"):\n",
    "    W = tf.Variable(tf.truncated_normal([100, relation_classes], -0.1, 0.1), name=\"W\")\n",
    "    b = tf.Variable(tf.zeros([relation_classes]), name=\"b\")\n",
    "    logits = tf.matmul(y_hidden_layer_drop, W) + b\n",
    "    predictions = tf.argmax(logits, 1)\n",
    "\n",
    "tv_all = tf.trainable_variables()\n",
    "tv_regu = []\n",
    "non_reg = [\"word_embedding/W:0\",\"pos_embedding/W:0\",'dep_embedding/W:0',\"global_step:0\",'hidden_layer/b:0','softmax_layer/b:0']\n",
    "for t in tv_all:\n",
    "    if t.name not in non_reg:\n",
    "        if(t.name.find('biases')==-1):\n",
    "            tv_regu.append(t)\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    l2_loss = lambda_l2 * tf.reduce_sum([ tf.nn.l2_loss(v) for v in tv_regu ])\n",
    "    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y))\n",
    "    total_loss = loss + l2_loss\n",
    "\n",
    "global_step = tf.Variable(0, name=\"global_step\")\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(0.001).minimize(total_loss, global_step=global_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open(data_dir + '/vocab.pkl', 'rb')\n",
    "vocab = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word2id = dict((w, i) for i,w in enumerate(vocab))\n",
    "id2word = dict((i, w) for i,w in enumerate(vocab))\n",
    "\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "word2id[unknown_token] = word_vocab_size -1\n",
    "id2word[word_vocab_size-1] = unknown_token\n",
    "\n",
    "pos_tags_vocab = []\n",
    "for line in open(data_dir + '/pos_tags.txt'):\n",
    "        pos_tags_vocab.append(line.strip())\n",
    "\n",
    "dep_vocab = []\n",
    "for line in open(data_dir + '/dependency_types.txt'):\n",
    "    dep_vocab.append(line.strip())\n",
    "\n",
    "relation_vocab = []\n",
    "for line in open(data_dir + '/relation_types.txt'):\n",
    "    relation_vocab.append(line.strip())\n",
    "\n",
    "\n",
    "rel2id = dict((w, i) for i,w in enumerate(relation_vocab))\n",
    "id2rel = dict((i, w) for i,w in enumerate(relation_vocab))\n",
    "\n",
    "pos_tag2id = dict((w, i) for i,w in enumerate(pos_tags_vocab))\n",
    "id2pos_tag = dict((i, w) for i,w in enumerate(pos_tags_vocab))\n",
    "\n",
    "dep2id = dict((w, i) for i,w in enumerate(dep_vocab))\n",
    "id2dep = dict((i, w) for i,w in enumerate(dep_vocab))\n",
    "\n",
    "pos_tag2id['OTH'] = 9\n",
    "id2pos_tag[9] = 'OTH'\n",
    "\n",
    "dep2id['OTH'] = 20\n",
    "id2dep[20] = 'OTH'\n",
    "\n",
    "JJ_pos_tags = ['JJ', 'JJR', 'JJS']\n",
    "NN_pos_tags = ['NN', 'NNS', 'NNP', 'NNPS']\n",
    "RB_pos_tags = ['RB', 'RBR', 'RBS']\n",
    "PRP_pos_tags = ['PRP', 'PRP$']\n",
    "VB_pos_tags = ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']\n",
    "_pos_tags = ['CC', 'CD', 'DT', 'IN']\n",
    "\n",
    "def pos_tag(x):\n",
    "    if x in JJ_pos_tags:\n",
    "        return pos_tag2id['JJ']\n",
    "    if x in NN_pos_tags:\n",
    "        return pos_tag2id['NN']\n",
    "    if x in RB_pos_tags:\n",
    "        return pos_tag2id['RB']\n",
    "    if x in PRP_pos_tags:\n",
    "        return pos_tag2id['PRP']\n",
    "    if x in VB_pos_tags:\n",
    "        return pos_tag2id['VB']\n",
    "    if x in _pos_tags:\n",
    "        return pos_tag2id[x]\n",
    "    else:\n",
    "        return 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# model = tf.train.latest_checkpoint(model_dir)\n",
    "# saver.restore(sess, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# f = open('data/word_embedding', 'rb')\n",
    "# word_embedding = pickle.load(f)\n",
    "# f.close()\n",
    "# sess.run(embedding_init, feed_dict={embedding_placeholder:word_embedding})\n",
    "# word_embedding_saver.save(sess, word_embd_dir + '/word_embd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "latest_embd = tf.train.latest_checkpoint(word_embd_dir)\n",
    "word_embedding_saver.restore(sess, latest_embd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open(data_dir + '/train_paths', 'rb')\n",
    "word_p1, word_p2, dep_p1, dep_p2, pos_p1, pos_p2 = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "relations = []\n",
    "for line in open(data_dir + '/train_relations.txt'):\n",
    "    relations.append(line.strip().split()[1])\n",
    "\n",
    "length = len(word_p1)\n",
    "num_batches = int(length/batch_size)\n",
    "\n",
    "for i in range(length):\n",
    "    for j, word in enumerate(word_p1[i]):\n",
    "        word = word.lower()\n",
    "        word_p1[i][j] = word if word in word2id else unknown_token \n",
    "    for k, word in enumerate(word_p2[i]):\n",
    "        word = word.lower()\n",
    "        word_p2[i][k] = word if word in word2id else unknown_token \n",
    "    for l, d in enumerate(dep_p1[i]):\n",
    "        dep_p1[i][l] = d if d in dep2id else 'OTH'\n",
    "    for m, d in enumerate(dep_p2[i]):\n",
    "        dep_p2[i][m] = d if d in dep2id else 'OTH'\n",
    "\n",
    "word_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "word_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "pos_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "pos_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "dep_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "dep_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "rel_ids = np.array([rel2id[rel] for rel in relations])\n",
    "path1_len = np.array([len(w) for w in word_p1], dtype=int)\n",
    "path2_len = np.array([len(w) for w in word_p2])\n",
    "\n",
    "for i in range(length):\n",
    "    for j, w in enumerate(word_p1[i]):\n",
    "        word_p1_ids[i][j] = word2id[w]\n",
    "    for j, w in enumerate(word_p2[i]):\n",
    "        word_p2_ids[i][j] = word2id[w]\n",
    "    for j, w in enumerate(pos_p1[i]):\n",
    "        pos_p1_ids[i][j] = pos_tag(w)\n",
    "    for j, w in enumerate(pos_p2[i]):\n",
    "        pos_p2_ids[i][j] = pos_tag(w)\n",
    "    for j, w in enumerate(dep_p1[i]):\n",
    "        dep_p1_ids[i][j] = dep2id[w]\n",
    "    for j, w in enumerate(dep_p2[i]):\n",
    "        dep_p2_ids[i][j] = dep2id[w]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 32010 loss: 0.279179\n",
      "Step: 32020 loss: 0.450493\n",
      "Step: 32030 loss: 0.67678\n",
      "Step: 32040 loss: 0.530038\n",
      "Step: 32050 loss: 0.18954\n",
      "Step: 32060 loss: 0.345747\n",
      "Step: 32070 loss: 0.399547\n",
      "Step: 32080 loss: 0.307604\n",
      "Step: 32090 loss: 0.211703\n",
      "Step: 32100 loss: 0.0819739\n",
      "Step: 32110 loss: 1.1188\n",
      "Step: 32120 loss: 0.82525\n",
      "Step: 32130 loss: 0.202954\n",
      "Step: 32140 loss: 0.0546259\n",
      "Step: 32150 loss: 0.1748\n",
      "Step: 32160 loss: 0.652707\n",
      "Step: 32170 loss: 0.0409834\n",
      "Step: 32180 loss: 1.07589\n",
      "Step: 32190 loss: 0.835989\n",
      "Step: 32200 loss: 0.306376\n",
      "Step: 32210 loss: 0.318982\n",
      "Step: 32220 loss: 0.604181\n",
      "Step: 32230 loss: 0.468826\n",
      "Step: 32240 loss: 0.15929\n",
      "Step: 32250 loss: 1.04704\n",
      "Step: 32260 loss: 0.175985\n",
      "Step: 32270 loss: 0.191001\n",
      "Step: 32280 loss: 0.0813737\n",
      "Step: 32290 loss: 0.124923\n",
      "Step: 32300 loss: 0.0491613\n",
      "Step: 32310 loss: 0.139159\n",
      "Step: 32320 loss: 0.0837461\n",
      "Step: 32330 loss: 0.264151\n",
      "Step: 32340 loss: 0.361466\n",
      "Step: 32350 loss: 0.361493\n",
      "Step: 32360 loss: 0.524457\n",
      "Step: 32370 loss: 0.0660119\n",
      "Step: 32380 loss: 0.0614393\n",
      "Step: 32390 loss: 1.12284\n",
      "Step: 32400 loss: 0.647188\n",
      "Step: 32410 loss: 0.361705\n",
      "Step: 32420 loss: 0.0945349\n",
      "Step: 32430 loss: 0.0846694\n",
      "Step: 32440 loss: 0.339376\n",
      "Step: 32450 loss: 0.0695926\n",
      "Step: 32460 loss: 0.218822\n",
      "Step: 32470 loss: 0.567043\n",
      "Step: 32480 loss: 0.310573\n",
      "Step: 32490 loss: 0.0524711\n",
      "Step: 32500 loss: 0.807416\n",
      "Step: 32510 loss: 0.345942\n",
      "Step: 32520 loss: 0.298481\n",
      "Step: 32530 loss: 0.20884\n",
      "Step: 32540 loss: 0.100555\n",
      "Step: 32550 loss: 0.553311\n",
      "Step: 32560 loss: 0.160225\n",
      "Step: 32570 loss: 0.45792\n",
      "Step: 32580 loss: 0.295235\n",
      "Step: 32590 loss: 0.65137\n",
      "Step: 32600 loss: 0.143354\n",
      "Step: 32610 loss: 0.372046\n",
      "Step: 32620 loss: 0.25772\n",
      "Step: 32630 loss: 0.556671\n",
      "Step: 32640 loss: 0.217369\n",
      "Step: 32650 loss: 0.215576\n",
      "Step: 32660 loss: 0.227676\n",
      "Step: 32670 loss: 0.252348\n",
      "Step: 32680 loss: 0.28657\n",
      "Step: 32690 loss: 0.247807\n",
      "Step: 32700 loss: 0.491792\n",
      "Step: 32710 loss: 0.678799\n",
      "Step: 32720 loss: 0.120787\n",
      "Step: 32730 loss: 1.08597\n",
      "Step: 32740 loss: 1.23383\n",
      "Step: 32750 loss: 0.186683\n",
      "Step: 32760 loss: 0.128247\n",
      "Step: 32770 loss: 0.257649\n",
      "Step: 32780 loss: 0.485908\n",
      "Step: 32790 loss: 0.253798\n",
      "Step: 32800 loss: 0.766357\n",
      "Step: 32810 loss: 0.41298\n",
      "Step: 32820 loss: 0.339788\n",
      "Step: 32830 loss: 0.269415\n",
      "Step: 32840 loss: 0.604805\n",
      "Step: 32850 loss: 0.582682\n",
      "Step: 32860 loss: 0.168542\n",
      "Step: 32870 loss: 0.113498\n",
      "Step: 32880 loss: 0.185727\n",
      "Step: 32890 loss: 0.174553\n",
      "Step: 32900 loss: 0.708578\n",
      "Step: 32910 loss: 0.220889\n",
      "Step: 32920 loss: 0.790742\n",
      "Step: 32930 loss: 0.109274\n",
      "Step: 32940 loss: 0.158193\n",
      "Step: 32950 loss: 0.141264\n",
      "Step: 32960 loss: 1.22535\n",
      "Step: 32970 loss: 0.0869452\n",
      "Step: 32980 loss: 0.704128\n",
      "Step: 32990 loss: 0.780896\n",
      "Step: 33000 loss: 0.129654\n",
      "Saved Model\n",
      "Step: 33010 loss: 0.0519998\n",
      "Step: 33020 loss: 0.252578\n",
      "Step: 33030 loss: 0.166322\n",
      "Step: 33040 loss: 0.588052\n",
      "Step: 33050 loss: 0.261396\n",
      "Step: 33060 loss: 0.204869\n",
      "Step: 33070 loss: 0.196828\n",
      "Step: 33080 loss: 0.138233\n",
      "Step: 33090 loss: 0.440589\n",
      "Step: 33100 loss: 0.218672\n",
      "Step: 33110 loss: 0.423693\n",
      "Step: 33120 loss: 0.12163\n",
      "Step: 33130 loss: 0.134871\n",
      "Step: 33140 loss: 0.0797675\n",
      "Step: 33150 loss: 1.03256\n",
      "Step: 33160 loss: 0.097784\n",
      "Step: 33170 loss: 0.0453078\n",
      "Step: 33180 loss: 0.0407143\n",
      "Step: 33190 loss: 0.159895\n",
      "Step: 33200 loss: 0.225456\n",
      "Step: 33210 loss: 0.229238\n",
      "Step: 33220 loss: 0.334664\n",
      "Step: 33230 loss: 0.057922\n",
      "Step: 33240 loss: 0.589104\n",
      "Step: 33250 loss: 0.620646\n",
      "Step: 33260 loss: 0.137862\n",
      "Step: 33270 loss: 0.500146\n",
      "Step: 33280 loss: 0.489662\n",
      "Step: 33290 loss: 0.0648951\n",
      "Step: 33300 loss: 0.354633\n",
      "Step: 33310 loss: 0.16308\n",
      "Step: 33320 loss: 0.553967\n",
      "Step: 33330 loss: 0.158187\n",
      "Step: 33340 loss: 0.388153\n",
      "Step: 33350 loss: 0.618726\n",
      "Step: 33360 loss: 0.359482\n",
      "Step: 33370 loss: 0.403872\n",
      "Step: 33380 loss: 0.5658\n",
      "Step: 33390 loss: 0.434459\n",
      "Step: 33400 loss: 0.297456\n",
      "Step: 33410 loss: 0.297703\n",
      "Step: 33420 loss: 0.399008\n",
      "Step: 33430 loss: 0.144178\n",
      "Step: 33440 loss: 0.16621\n",
      "Step: 33450 loss: 0.478584\n",
      "Step: 33460 loss: 0.498657\n",
      "Step: 33470 loss: 0.190341\n",
      "Step: 33480 loss: 0.352795\n",
      "Step: 33490 loss: 0.212605\n",
      "Step: 33500 loss: 0.202485\n",
      "Step: 33510 loss: 0.0401946\n",
      "Step: 33520 loss: 0.0721108\n",
      "Step: 33530 loss: 1.80514\n",
      "Step: 33540 loss: 0.446917\n",
      "Step: 33550 loss: 0.156792\n",
      "Step: 33560 loss: 0.51696\n",
      "Step: 33570 loss: 0.533041\n",
      "Step: 33580 loss: 1.56295\n",
      "Step: 33590 loss: 0.196508\n",
      "Step: 33600 loss: 0.526751\n",
      "Step: 33610 loss: 0.407434\n",
      "Step: 33620 loss: 0.797104\n",
      "Step: 33630 loss: 0.292795\n",
      "Step: 33640 loss: 0.932842\n",
      "Step: 33650 loss: 0.343432\n",
      "Step: 33660 loss: 0.210032\n",
      "Step: 33670 loss: 0.454733\n",
      "Step: 33680 loss: 0.652721\n",
      "Step: 33690 loss: 0.063564\n",
      "Step: 33700 loss: 0.172232\n",
      "Step: 33710 loss: 0.539137\n",
      "Step: 33720 loss: 0.506651\n",
      "Step: 33730 loss: 0.323335\n",
      "Step: 33740 loss: 0.0870652\n",
      "Step: 33750 loss: 0.111432\n",
      "Step: 33760 loss: 0.818661\n",
      "Step: 33770 loss: 0.150225\n",
      "Step: 33780 loss: 0.540508\n",
      "Step: 33790 loss: 0.0757697\n",
      "Step: 33800 loss: 0.606815\n",
      "Step: 33810 loss: 0.154748\n",
      "Step: 33820 loss: 0.0508892\n",
      "Step: 33830 loss: 0.348589\n",
      "Step: 33840 loss: 0.303386\n",
      "Step: 33850 loss: 0.141997\n",
      "Step: 33860 loss: 0.26169\n",
      "Step: 33870 loss: 0.0708484\n",
      "Step: 33880 loss: 0.0763586\n",
      "Step: 33890 loss: 0.384527\n",
      "Step: 33900 loss: 0.0617894\n",
      "Step: 33910 loss: 0.253305\n",
      "Step: 33920 loss: 0.42618\n",
      "Step: 33930 loss: 0.173075\n",
      "Step: 33940 loss: 0.0755884\n",
      "Step: 33950 loss: 0.727628\n",
      "Step: 33960 loss: 0.110905\n",
      "Step: 33970 loss: 0.135607\n",
      "Step: 33980 loss: 0.270418\n",
      "Step: 33990 loss: 0.502588\n",
      "Step: 34000 loss: 0.852212\n",
      "Saved Model\n",
      "Step: 34010 loss: 0.339386\n",
      "Step: 34020 loss: 0.255453\n",
      "Step: 34030 loss: 0.288253\n",
      "Step: 34040 loss: 0.0478743\n",
      "Step: 34050 loss: 0.22066\n",
      "Step: 34060 loss: 0.117599\n",
      "Step: 34070 loss: 0.688173\n",
      "Step: 34080 loss: 0.260392\n",
      "Step: 34090 loss: 0.53858\n",
      "Step: 34100 loss: 0.572392\n",
      "Step: 34110 loss: 0.249442\n",
      "Step: 34120 loss: 0.506578\n",
      "Step: 34130 loss: 0.312098\n",
      "Step: 34140 loss: 0.426077\n",
      "Step: 34150 loss: 0.596476\n",
      "Step: 34160 loss: 0.424022\n",
      "Step: 34170 loss: 0.853512\n",
      "Step: 34180 loss: 0.346202\n",
      "Step: 34190 loss: 0.137192\n",
      "Step: 34200 loss: 1.16311\n",
      "Step: 34210 loss: 0.334048\n",
      "Step: 34220 loss: 0.198114\n",
      "Step: 34230 loss: 0.455139\n",
      "Step: 34240 loss: 0.394255\n",
      "Step: 34250 loss: 0.0740821\n",
      "Step: 34260 loss: 0.0334986\n",
      "Step: 34270 loss: 0.237459\n",
      "Step: 34280 loss: 0.822495\n",
      "Step: 34290 loss: 0.235822\n",
      "Step: 34300 loss: 0.158782\n",
      "Step: 34310 loss: 0.337128\n",
      "Step: 34320 loss: 0.0710535\n",
      "Step: 34330 loss: 0.958673\n",
      "Step: 34340 loss: 1.20052\n",
      "Step: 34350 loss: 0.0966727\n",
      "Step: 34360 loss: 0.271976\n",
      "Step: 34370 loss: 0.686194\n",
      "Step: 34380 loss: 0.329945\n",
      "Step: 34390 loss: 0.68813\n",
      "Step: 34400 loss: 0.306913\n",
      "Step: 34410 loss: 0.125988\n",
      "Step: 34420 loss: 0.944902\n",
      "Step: 34430 loss: 0.325162\n",
      "Step: 34440 loss: 0.250958\n",
      "Step: 34450 loss: 0.563147\n",
      "Step: 34460 loss: 0.105593\n",
      "Step: 34470 loss: 0.134505\n",
      "Step: 34480 loss: 0.90247\n",
      "Step: 34490 loss: 0.0611105\n",
      "Step: 34500 loss: 0.464936\n",
      "Step: 34510 loss: 0.532991\n",
      "Step: 34520 loss: 0.431769\n",
      "Step: 34530 loss: 0.0611431\n",
      "Step: 34540 loss: 0.032559\n",
      "Step: 34550 loss: 0.228357\n",
      "Step: 34560 loss: 1.41214\n",
      "Step: 34570 loss: 0.324937\n",
      "Step: 34580 loss: 0.52039\n",
      "Step: 34590 loss: 0.234365\n",
      "Step: 34600 loss: 0.108633\n",
      "Step: 34610 loss: 0.0390641\n",
      "Step: 34620 loss: 0.0377687\n",
      "Step: 34630 loss: 0.278421\n",
      "Step: 34640 loss: 0.355867\n",
      "Step: 34650 loss: 0.145756\n",
      "Step: 34660 loss: 0.251277\n",
      "Step: 34670 loss: 0.237728\n",
      "Step: 34680 loss: 0.136915\n",
      "Step: 34690 loss: 0.281355\n",
      "Step: 34700 loss: 0.168524\n",
      "Step: 34710 loss: 0.131746\n",
      "Step: 34720 loss: 0.041946\n",
      "Step: 34730 loss: 0.205354\n",
      "Step: 34740 loss: 0.130313\n",
      "Step: 34750 loss: 0.976827\n",
      "Step: 34760 loss: 0.227205\n",
      "Step: 34770 loss: 0.434699\n",
      "Step: 34780 loss: 0.1137\n",
      "Step: 34790 loss: 0.783135\n",
      "Step: 34800 loss: 0.261772\n",
      "Step: 34810 loss: 0.485101\n",
      "Step: 34820 loss: 0.456713\n",
      "Step: 34830 loss: 0.177704\n",
      "Step: 34840 loss: 0.0602775\n",
      "Step: 34850 loss: 0.17149\n",
      "Step: 34860 loss: 0.0890281\n",
      "Step: 34870 loss: 0.492132\n",
      "Step: 34880 loss: 0.999579\n",
      "Step: 34890 loss: 0.296464\n",
      "Step: 34900 loss: 0.202356\n",
      "Step: 34910 loss: 0.598766\n",
      "Step: 34920 loss: 0.192588\n",
      "Step: 34930 loss: 0.102359\n",
      "Step: 34940 loss: 0.27556\n",
      "Step: 34950 loss: 0.529312\n",
      "Step: 34960 loss: 0.337119\n",
      "Step: 34970 loss: 0.396434\n",
      "Step: 34980 loss: 0.137415\n",
      "Step: 34990 loss: 0.835906\n",
      "Step: 35000 loss: 0.869232\n",
      "Saved Model\n",
      "Step: 35010 loss: 0.0789146\n",
      "Step: 35020 loss: 0.167259\n",
      "Step: 35030 loss: 0.0617612\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 35040 loss: 0.479934\n",
      "Step: 35050 loss: 0.0341393\n",
      "Step: 35060 loss: 0.215971\n",
      "Step: 35070 loss: 0.646517\n",
      "Step: 35080 loss: 0.262261\n",
      "Step: 35090 loss: 0.0584262\n",
      "Step: 35100 loss: 0.0881525\n",
      "Step: 35110 loss: 0.297986\n",
      "Step: 35120 loss: 0.207136\n",
      "Step: 35130 loss: 1.25428\n",
      "Step: 35140 loss: 1.44878\n",
      "Step: 35150 loss: 0.375991\n",
      "Step: 35160 loss: 0.0596189\n",
      "Step: 35170 loss: 0.712018\n",
      "Step: 35180 loss: 0.750539\n",
      "Step: 35190 loss: 0.18919\n",
      "Step: 35200 loss: 0.0566818\n",
      "Step: 35210 loss: 0.0729581\n",
      "Step: 35220 loss: 0.222028\n",
      "Step: 35230 loss: 0.498812\n",
      "Step: 35240 loss: 0.496733\n",
      "Step: 35250 loss: 0.0773993\n",
      "Step: 35260 loss: 0.0732535\n",
      "Step: 35270 loss: 0.0512249\n",
      "Step: 35280 loss: 0.412121\n",
      "Step: 35290 loss: 0.247884\n",
      "Step: 35300 loss: 0.0792624\n",
      "Step: 35310 loss: 0.299243\n",
      "Step: 35320 loss: 1.06095\n",
      "Step: 35330 loss: 0.139944\n",
      "Step: 35340 loss: 0.126545\n",
      "Step: 35350 loss: 0.062601\n",
      "Step: 35360 loss: 0.452786\n",
      "Step: 35370 loss: 0.0500685\n",
      "Step: 35380 loss: 0.888264\n",
      "Step: 35390 loss: 0.173906\n",
      "Step: 35400 loss: 0.0560225\n",
      "Step: 35410 loss: 0.0925706\n",
      "Step: 35420 loss: 0.0626682\n",
      "Step: 35430 loss: 0.103716\n",
      "Step: 35440 loss: 0.114921\n",
      "Step: 35450 loss: 0.0709133\n",
      "Step: 35460 loss: 0.0916893\n",
      "Step: 35470 loss: 0.50135\n",
      "Step: 35480 loss: 0.311349\n",
      "Step: 35490 loss: 0.092508\n",
      "Step: 35500 loss: 0.0405589\n",
      "Step: 35510 loss: 0.28047\n",
      "Step: 35520 loss: 0.185782\n",
      "Step: 35530 loss: 0.150092\n",
      "Step: 35540 loss: 0.478324\n",
      "Step: 35550 loss: 1.86648\n",
      "Step: 35560 loss: 0.811279\n",
      "Step: 35570 loss: 0.27852\n",
      "Step: 35580 loss: 0.148737\n",
      "Step: 35590 loss: 0.249314\n",
      "Step: 35600 loss: 0.509873\n",
      "Step: 35610 loss: 0.156209\n",
      "Step: 35620 loss: 0.107887\n",
      "Step: 35630 loss: 0.159447\n",
      "Step: 35640 loss: 0.149809\n",
      "Step: 35650 loss: 0.117096\n",
      "Step: 35660 loss: 0.0359255\n",
      "Step: 35670 loss: 0.382418\n",
      "Step: 35680 loss: 0.129737\n",
      "Step: 35690 loss: 0.12679\n",
      "Step: 35700 loss: 0.599195\n",
      "Step: 35710 loss: 0.574362\n",
      "Step: 35720 loss: 0.202478\n",
      "Step: 35730 loss: 0.184996\n",
      "Step: 35740 loss: 0.139286\n",
      "Step: 35750 loss: 0.245985\n",
      "Step: 35760 loss: 0.235978\n",
      "Step: 35770 loss: 0.259779\n",
      "Step: 35780 loss: 0.0521587\n",
      "Step: 35790 loss: 0.603587\n",
      "Step: 35800 loss: 0.225551\n",
      "Step: 35810 loss: 0.111661\n",
      "Step: 35820 loss: 0.0833981\n",
      "Step: 35830 loss: 0.409975\n",
      "Step: 35840 loss: 0.347659\n",
      "Step: 35850 loss: 0.128399\n",
      "Step: 35860 loss: 0.0713995\n",
      "Step: 35870 loss: 0.158054\n",
      "Step: 35880 loss: 0.0822342\n",
      "Step: 35890 loss: 0.435791\n",
      "Step: 35900 loss: 0.0787529\n",
      "Step: 35910 loss: 0.587082\n",
      "Step: 35920 loss: 0.0469604\n",
      "Step: 35930 loss: 0.47132\n",
      "Step: 35940 loss: 0.288581\n",
      "Step: 35950 loss: 0.752205\n",
      "Step: 35960 loss: 0.145229\n",
      "Step: 35970 loss: 0.264283\n",
      "Step: 35980 loss: 1.195\n",
      "Step: 35990 loss: 0.346844\n",
      "Step: 36000 loss: 0.239945\n",
      "Saved Model\n",
      "Step: 36010 loss: 0.1021\n",
      "Step: 36020 loss: 0.237226\n",
      "Step: 36030 loss: 0.573529\n",
      "Step: 36040 loss: 0.208757\n",
      "Step: 36050 loss: 0.126912\n",
      "Step: 36060 loss: 0.065058\n",
      "Step: 36070 loss: 0.318923\n",
      "Step: 36080 loss: 0.102025\n",
      "Step: 36090 loss: 0.337071\n",
      "Step: 36100 loss: 0.142\n",
      "Step: 36110 loss: 0.54326\n",
      "Step: 36120 loss: 1.26277\n",
      "Step: 36130 loss: 0.126428\n",
      "Step: 36140 loss: 0.0480947\n",
      "Step: 36150 loss: 0.123598\n",
      "Step: 36160 loss: 0.959337\n",
      "Step: 36170 loss: 0.0967006\n",
      "Step: 36180 loss: 0.361327\n",
      "Step: 36190 loss: 0.181776\n",
      "Step: 36200 loss: 0.0820338\n",
      "Step: 36210 loss: 0.0560747\n",
      "Step: 36220 loss: 0.0899908\n",
      "Step: 36230 loss: 0.367896\n",
      "Step: 36240 loss: 0.116067\n",
      "Step: 36250 loss: 0.234468\n",
      "Step: 36260 loss: 0.12646\n",
      "Step: 36270 loss: 0.0740225\n",
      "Step: 36280 loss: 0.169993\n",
      "Step: 36290 loss: 0.584961\n",
      "Step: 36300 loss: 0.0750111\n",
      "Step: 36310 loss: 0.0473489\n",
      "Step: 36320 loss: 0.0789362\n",
      "Step: 36330 loss: 0.20172\n",
      "Step: 36340 loss: 0.133674\n",
      "Step: 36350 loss: 0.362497\n",
      "Step: 36360 loss: 0.0503498\n",
      "Step: 36370 loss: 0.0591355\n",
      "Step: 36380 loss: 0.153055\n",
      "Step: 36390 loss: 1.43472\n",
      "Step: 36400 loss: 0.152052\n",
      "Step: 36410 loss: 0.246163\n",
      "Step: 36420 loss: 0.402833\n",
      "Step: 36430 loss: 0.0851191\n",
      "Step: 36440 loss: 0.0656493\n",
      "Step: 36450 loss: 0.051456\n",
      "Step: 36460 loss: 0.119649\n",
      "Step: 36470 loss: 0.250625\n",
      "Step: 36480 loss: 0.263945\n",
      "Step: 36490 loss: 0.570724\n",
      "Step: 36500 loss: 0.568354\n",
      "Step: 36510 loss: 0.416346\n",
      "Step: 36520 loss: 0.0889314\n",
      "Step: 36530 loss: 0.124682\n",
      "Step: 36540 loss: 0.403769\n",
      "Step: 36550 loss: 0.16202\n",
      "Step: 36560 loss: 0.501903\n",
      "Step: 36570 loss: 1.28176\n",
      "Step: 36580 loss: 0.570956\n",
      "Step: 36590 loss: 0.502472\n",
      "Step: 36600 loss: 0.29656\n",
      "Step: 36610 loss: 0.0792443\n",
      "Step: 36620 loss: 0.234697\n",
      "Step: 36630 loss: 0.171989\n",
      "Step: 36640 loss: 0.14538\n",
      "Step: 36650 loss: 0.142143\n",
      "Step: 36660 loss: 0.223581\n",
      "Step: 36670 loss: 0.340763\n",
      "Step: 36680 loss: 0.068054\n",
      "Step: 36690 loss: 0.0898953\n",
      "Step: 36700 loss: 0.0456919\n",
      "Step: 36710 loss: 0.132188\n",
      "Step: 36720 loss: 0.126349\n",
      "Step: 36730 loss: 0.437822\n",
      "Step: 36740 loss: 0.855771\n",
      "Step: 36750 loss: 0.143884\n",
      "Step: 36760 loss: 0.516584\n",
      "Step: 36770 loss: 0.629024\n",
      "Step: 36780 loss: 0.355114\n",
      "Step: 36790 loss: 0.225381\n",
      "Step: 36800 loss: 0.507348\n",
      "Step: 36810 loss: 0.15247\n",
      "Step: 36820 loss: 0.307558\n",
      "Step: 36830 loss: 0.46234\n",
      "Step: 36840 loss: 0.357648\n",
      "Step: 36850 loss: 0.616753\n",
      "Step: 36860 loss: 0.0341869\n",
      "Step: 36870 loss: 0.189619\n",
      "Step: 36880 loss: 0.0708052\n",
      "Step: 36890 loss: 0.438577\n",
      "Step: 36900 loss: 0.107736\n",
      "Step: 36910 loss: 0.589731\n",
      "Step: 36920 loss: 1.05829\n",
      "Step: 36930 loss: 0.122172\n",
      "Step: 36940 loss: 0.0866082\n",
      "Step: 36950 loss: 0.0456612\n",
      "Step: 36960 loss: 0.475289\n",
      "Step: 36970 loss: 0.192894\n",
      "Step: 36980 loss: 0.860514\n",
      "Step: 36990 loss: 0.14634\n",
      "Step: 37000 loss: 0.0504587\n",
      "Saved Model\n",
      "Step: 37010 loss: 0.034614\n",
      "Step: 37020 loss: 0.0822538\n",
      "Step: 37030 loss: 0.157546\n",
      "Step: 37040 loss: 0.108547\n",
      "Step: 37050 loss: 0.42092\n",
      "Step: 37060 loss: 0.188831\n",
      "Step: 37070 loss: 0.0348117\n",
      "Step: 37080 loss: 0.0640449\n",
      "Step: 37090 loss: 0.0626833\n",
      "Step: 37100 loss: 0.0857915\n",
      "Step: 37110 loss: 0.202111\n",
      "Step: 37120 loss: 0.238651\n",
      "Step: 37130 loss: 0.458686\n",
      "Step: 37140 loss: 0.48106\n",
      "Step: 37150 loss: 0.698216\n",
      "Step: 37160 loss: 0.280815\n",
      "Step: 37170 loss: 0.311184\n",
      "Step: 37180 loss: 0.0801302\n",
      "Step: 37190 loss: 0.361621\n",
      "Step: 37200 loss: 0.816276\n",
      "Step: 37210 loss: 0.618614\n",
      "Step: 37220 loss: 0.170065\n",
      "Step: 37230 loss: 0.0613283\n",
      "Step: 37240 loss: 0.226215\n",
      "Step: 37250 loss: 0.13893\n",
      "Step: 37260 loss: 0.849102\n",
      "Step: 37270 loss: 0.283642\n",
      "Step: 37280 loss: 0.359286\n",
      "Step: 37290 loss: 0.0354955\n",
      "Step: 37300 loss: 0.238376\n",
      "Step: 37310 loss: 0.172081\n",
      "Step: 37320 loss: 0.067206\n",
      "Step: 37330 loss: 0.126519\n",
      "Step: 37340 loss: 0.240706\n",
      "Step: 37350 loss: 0.42799\n",
      "Step: 37360 loss: 0.144654\n",
      "Step: 37370 loss: 0.513961\n",
      "Step: 37380 loss: 0.398112\n",
      "Step: 37390 loss: 0.241537\n",
      "Step: 37400 loss: 0.0771881\n",
      "Step: 37410 loss: 0.0799709\n",
      "Step: 37420 loss: 0.0461238\n",
      "Step: 37430 loss: 0.051412\n",
      "Step: 37440 loss: 0.287318\n",
      "Step: 37450 loss: 0.0495305\n",
      "Step: 37460 loss: 0.0564129\n",
      "Step: 37470 loss: 0.382087\n",
      "Step: 37480 loss: 0.0790712\n",
      "Step: 37490 loss: 0.122997\n",
      "Step: 37500 loss: 0.253297\n",
      "Step: 37510 loss: 0.0811301\n",
      "Step: 37520 loss: 0.171803\n",
      "Step: 37530 loss: 0.672567\n",
      "Step: 37540 loss: 0.466559\n",
      "Step: 37550 loss: 0.0777753\n",
      "Step: 37560 loss: 0.173346\n",
      "Step: 37570 loss: 0.661633\n",
      "Step: 37580 loss: 0.361602\n",
      "Step: 37590 loss: 0.125056\n",
      "Step: 37600 loss: 0.480503\n",
      "Step: 37610 loss: 0.125931\n",
      "Step: 37620 loss: 0.843845\n",
      "Step: 37630 loss: 0.187214\n",
      "Step: 37640 loss: 0.574217\n",
      "Step: 37650 loss: 0.197172\n",
      "Step: 37660 loss: 0.102726\n",
      "Step: 37670 loss: 0.0965903\n",
      "Step: 37680 loss: 0.0583482\n",
      "Step: 37690 loss: 0.102608\n",
      "Step: 37700 loss: 0.38248\n",
      "Step: 37710 loss: 0.285302\n",
      "Step: 37720 loss: 1.00413\n",
      "Step: 37730 loss: 0.368856\n",
      "Step: 37740 loss: 0.0384795\n",
      "Step: 37750 loss: 0.261854\n",
      "Step: 37760 loss: 0.435868\n",
      "Step: 37770 loss: 0.371113\n",
      "Step: 37780 loss: 0.487939\n",
      "Step: 37790 loss: 0.218513\n",
      "Step: 37800 loss: 0.227342\n",
      "Step: 37810 loss: 0.0389586\n",
      "Step: 37820 loss: 0.127845\n",
      "Step: 37830 loss: 0.218821\n",
      "Step: 37840 loss: 0.837086\n",
      "Step: 37850 loss: 0.280632\n",
      "Step: 37860 loss: 0.310707\n",
      "Step: 37870 loss: 0.327278\n",
      "Step: 37880 loss: 0.272311\n",
      "Step: 37890 loss: 1.76168\n",
      "Step: 37900 loss: 0.156936\n",
      "Step: 37910 loss: 0.420932\n",
      "Step: 37920 loss: 0.233515\n",
      "Step: 37930 loss: 0.323751\n",
      "Step: 37940 loss: 0.267907\n",
      "Step: 37950 loss: 0.706508\n",
      "Step: 37960 loss: 0.485982\n",
      "Step: 37970 loss: 0.0675981\n",
      "Step: 37980 loss: 0.146002\n",
      "Step: 37990 loss: 0.250093\n",
      "Step: 38000 loss: 1.23034\n",
      "Saved Model\n",
      "Step: 38010 loss: 0.102561\n",
      "Step: 38020 loss: 0.43208\n",
      "Step: 38030 loss: 0.109157\n",
      "Step: 38040 loss: 0.0484882\n",
      "Step: 38050 loss: 0.106087\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 38060 loss: 0.286757\n",
      "Step: 38070 loss: 0.166176\n",
      "Step: 38080 loss: 0.295198\n",
      "Step: 38090 loss: 0.145652\n",
      "Step: 38100 loss: 0.526205\n",
      "Step: 38110 loss: 0.179935\n",
      "Step: 38120 loss: 0.255303\n",
      "Step: 38130 loss: 0.206683\n",
      "Step: 38140 loss: 0.316885\n",
      "Step: 38150 loss: 0.0736747\n",
      "Step: 38160 loss: 0.450026\n",
      "Step: 38170 loss: 0.657418\n",
      "Step: 38180 loss: 0.433982\n",
      "Step: 38190 loss: 0.639218\n",
      "Step: 38200 loss: 0.110932\n",
      "Step: 38210 loss: 0.288301\n",
      "Step: 38220 loss: 0.119983\n",
      "Step: 38230 loss: 0.356939\n",
      "Step: 38240 loss: 0.141395\n",
      "Step: 38250 loss: 0.0782554\n",
      "Step: 38260 loss: 0.0788708\n",
      "Step: 38270 loss: 0.0932686\n",
      "Step: 38280 loss: 0.244754\n",
      "Step: 38290 loss: 0.100126\n",
      "Step: 38300 loss: 0.0432423\n",
      "Step: 38310 loss: 0.424576\n",
      "Step: 38320 loss: 0.07325\n",
      "Step: 38330 loss: 0.349547\n",
      "Step: 38340 loss: 0.726019\n",
      "Step: 38350 loss: 0.171024\n",
      "Step: 38360 loss: 0.253216\n",
      "Step: 38370 loss: 0.800708\n",
      "Step: 38380 loss: 0.671277\n",
      "Step: 38390 loss: 0.296961\n",
      "Step: 38400 loss: 0.4337\n",
      "Step: 38410 loss: 0.0595922\n",
      "Step: 38420 loss: 0.494958\n",
      "Step: 38430 loss: 0.312978\n",
      "Step: 38440 loss: 0.100107\n",
      "Step: 38450 loss: 0.136195\n",
      "Step: 38460 loss: 0.181219\n",
      "Step: 38470 loss: 0.469768\n",
      "Step: 38480 loss: 0.298869\n",
      "Step: 38490 loss: 0.494485\n",
      "Step: 38500 loss: 0.194579\n",
      "Step: 38510 loss: 0.113923\n",
      "Step: 38520 loss: 0.984361\n",
      "Step: 38530 loss: 0.0385943\n",
      "Step: 38540 loss: 0.0896106\n",
      "Step: 38550 loss: 0.263959\n",
      "Step: 38560 loss: 0.567814\n",
      "Step: 38570 loss: 0.235059\n",
      "Step: 38580 loss: 0.272767\n",
      "Step: 38590 loss: 0.193031\n",
      "Step: 38600 loss: 0.0499388\n",
      "Step: 38610 loss: 0.0965627\n",
      "Step: 38620 loss: 0.0602252\n",
      "Step: 38630 loss: 0.531888\n",
      "Step: 38640 loss: 0.224971\n",
      "Step: 38650 loss: 0.0414868\n",
      "Step: 38660 loss: 0.0728302\n",
      "Step: 38670 loss: 0.113862\n",
      "Step: 38680 loss: 0.0482225\n",
      "Step: 38690 loss: 0.157435\n",
      "Step: 38700 loss: 0.0884899\n",
      "Step: 38710 loss: 0.103141\n",
      "Step: 38720 loss: 0.0490424\n",
      "Step: 38730 loss: 0.339228\n",
      "Step: 38740 loss: 0.799627\n",
      "Step: 38750 loss: 0.756167\n",
      "Step: 38760 loss: 0.406706\n",
      "Step: 38770 loss: 0.0888515\n",
      "Step: 38780 loss: 0.338716\n",
      "Step: 38790 loss: 0.265237\n",
      "Step: 38800 loss: 0.367745\n",
      "Step: 38810 loss: 0.256484\n",
      "Step: 38820 loss: 0.0848168\n",
      "Step: 38830 loss: 0.0651632\n",
      "Step: 38840 loss: 0.0980495\n",
      "Step: 38850 loss: 0.0355474\n",
      "Step: 38860 loss: 0.260777\n",
      "Step: 38870 loss: 0.93319\n",
      "Step: 38880 loss: 0.221613\n",
      "Step: 38890 loss: 0.198349\n",
      "Step: 38900 loss: 0.0963494\n",
      "Step: 38910 loss: 0.311715\n",
      "Step: 38920 loss: 0.0849332\n",
      "Step: 38930 loss: 0.122816\n",
      "Step: 38940 loss: 0.110851\n",
      "Step: 38950 loss: 0.419569\n",
      "Step: 38960 loss: 0.251236\n",
      "Step: 38970 loss: 0.325968\n",
      "Step: 38980 loss: 0.679176\n",
      "Step: 38990 loss: 0.591901\n",
      "Step: 39000 loss: 0.234016\n",
      "Saved Model\n",
      "Step: 39010 loss: 0.0545604\n",
      "Step: 39020 loss: 0.0912514\n",
      "Step: 39030 loss: 0.292736\n",
      "Step: 39040 loss: 0.251766\n",
      "Step: 39050 loss: 0.267657\n",
      "Step: 39060 loss: 0.230534\n",
      "Step: 39070 loss: 0.128905\n",
      "Step: 39080 loss: 0.0549112\n",
      "Step: 39090 loss: 0.0642834\n",
      "Step: 39100 loss: 0.0623818\n",
      "Step: 39110 loss: 0.191809\n",
      "Step: 39120 loss: 0.128399\n",
      "Step: 39130 loss: 0.51208\n",
      "Step: 39140 loss: 0.763994\n",
      "Step: 39150 loss: 0.0812067\n",
      "Step: 39160 loss: 0.509411\n",
      "Step: 39170 loss: 0.454004\n",
      "Step: 39180 loss: 0.358377\n",
      "Step: 39190 loss: 0.305999\n",
      "Step: 39200 loss: 0.502548\n",
      "Step: 39210 loss: 0.179754\n",
      "Step: 39220 loss: 0.400112\n",
      "Step: 39230 loss: 0.255935\n",
      "Step: 39240 loss: 0.659457\n",
      "Step: 39250 loss: 0.452684\n",
      "Step: 39260 loss: 0.235141\n",
      "Step: 39270 loss: 0.0817124\n",
      "Step: 39280 loss: 0.118241\n",
      "Step: 39290 loss: 0.13631\n",
      "Step: 39300 loss: 0.188971\n",
      "Step: 39310 loss: 0.162172\n",
      "Step: 39320 loss: 1.40928\n",
      "Step: 39330 loss: 0.0616058\n",
      "Step: 39340 loss: 0.0490121\n",
      "Step: 39350 loss: 0.181222\n",
      "Step: 39360 loss: 0.570388\n",
      "Step: 39370 loss: 0.173229\n",
      "Step: 39380 loss: 1.24212\n",
      "Step: 39390 loss: 0.0999865\n",
      "Step: 39400 loss: 0.396562\n",
      "Step: 39410 loss: 0.144778\n",
      "Step: 39420 loss: 0.140412\n",
      "Step: 39430 loss: 0.566947\n",
      "Step: 39440 loss: 0.141489\n",
      "Step: 39450 loss: 0.238646\n",
      "Step: 39460 loss: 0.239009\n",
      "Step: 39470 loss: 0.372201\n",
      "Step: 39480 loss: 0.140502\n",
      "Step: 39490 loss: 0.410635\n",
      "Step: 39500 loss: 0.0754473\n",
      "Step: 39510 loss: 0.239318\n",
      "Step: 39520 loss: 0.140752\n",
      "Step: 39530 loss: 0.396113\n",
      "Step: 39540 loss: 0.412016\n",
      "Step: 39550 loss: 0.0776534\n",
      "Step: 39560 loss: 0.276814\n",
      "Step: 39570 loss: 0.0822869\n",
      "Step: 39580 loss: 0.141112\n",
      "Step: 39590 loss: 0.935769\n",
      "Step: 39600 loss: 0.179639\n",
      "Step: 39610 loss: 0.183977\n",
      "Step: 39620 loss: 0.0977226\n",
      "Step: 39630 loss: 0.2478\n",
      "Step: 39640 loss: 0.48316\n",
      "Step: 39650 loss: 0.087502\n",
      "Step: 39660 loss: 0.0837592\n",
      "Step: 39670 loss: 0.510398\n",
      "Step: 39680 loss: 0.0632234\n",
      "Step: 39690 loss: 0.0458402\n",
      "Step: 39700 loss: 0.18639\n",
      "Step: 39710 loss: 0.304674\n",
      "Step: 39720 loss: 0.303385\n",
      "Step: 39730 loss: 0.0461026\n",
      "Step: 39740 loss: 0.343872\n",
      "Step: 39750 loss: 0.314915\n",
      "Step: 39760 loss: 0.0993401\n",
      "Step: 39770 loss: 0.100445\n",
      "Step: 39780 loss: 0.145492\n",
      "Step: 39790 loss: 0.100532\n",
      "Step: 39800 loss: 0.244508\n",
      "Step: 39810 loss: 0.112447\n",
      "Step: 39820 loss: 0.0626062\n",
      "Step: 39830 loss: 0.132417\n",
      "Step: 39840 loss: 0.0859641\n",
      "Step: 39850 loss: 0.176337\n",
      "Step: 39860 loss: 0.0986162\n",
      "Step: 39870 loss: 0.534909\n",
      "Step: 39880 loss: 0.224312\n",
      "Step: 39890 loss: 0.0810062\n",
      "Step: 39900 loss: 0.277237\n",
      "Step: 39910 loss: 0.456735\n",
      "Step: 39920 loss: 0.0746093\n",
      "Step: 39930 loss: 0.37889\n",
      "Step: 39940 loss: 0.824266\n",
      "Step: 39950 loss: 0.120659\n",
      "Step: 39960 loss: 0.506527\n",
      "Step: 39970 loss: 0.203106\n",
      "Step: 39980 loss: 0.337298\n",
      "Step: 39990 loss: 0.0426005\n",
      "Step: 40000 loss: 1.02105\n",
      "Saved Model\n",
      "Step: 40010 loss: 0.0810066\n",
      "Step: 40020 loss: 0.966299\n",
      "Step: 40030 loss: 0.090048\n",
      "Step: 40040 loss: 0.666189\n",
      "Step: 40050 loss: 0.190541\n",
      "Step: 40060 loss: 0.0753128\n",
      "Step: 40070 loss: 0.0869654\n",
      "Step: 40080 loss: 0.0626881\n",
      "Step: 40090 loss: 0.0713051\n",
      "Step: 40100 loss: 0.232744\n",
      "Step: 40110 loss: 0.281767\n",
      "Step: 40120 loss: 0.744513\n",
      "Step: 40130 loss: 0.143158\n",
      "Step: 40140 loss: 0.138415\n",
      "Step: 40150 loss: 0.0637898\n",
      "Step: 40160 loss: 0.376352\n",
      "Step: 40170 loss: 0.100883\n",
      "Step: 40180 loss: 0.406032\n",
      "Step: 40190 loss: 0.14419\n",
      "Step: 40200 loss: 0.0772371\n",
      "Step: 40210 loss: 0.0477499\n",
      "Step: 40220 loss: 0.164815\n",
      "Step: 40230 loss: 0.2539\n",
      "Step: 40240 loss: 0.150313\n",
      "Step: 40250 loss: 0.518427\n",
      "Step: 40260 loss: 0.148654\n",
      "Step: 40270 loss: 0.664905\n",
      "Step: 40280 loss: 0.0870905\n",
      "Step: 40290 loss: 0.629594\n",
      "Step: 40300 loss: 0.0529834\n",
      "Step: 40310 loss: 0.0465315\n",
      "Step: 40320 loss: 0.405807\n",
      "Step: 40330 loss: 0.0418277\n",
      "Step: 40340 loss: 0.269774\n",
      "Step: 40350 loss: 0.587771\n",
      "Step: 40360 loss: 0.0894342\n",
      "Step: 40370 loss: 0.0983211\n",
      "Step: 40380 loss: 0.0507553\n",
      "Step: 40390 loss: 0.290747\n",
      "Step: 40400 loss: 0.424472\n",
      "Step: 40410 loss: 0.564431\n",
      "Step: 40420 loss: 0.40912\n",
      "Step: 40430 loss: 0.249212\n",
      "Step: 40440 loss: 0.0717456\n",
      "Step: 40450 loss: 0.387945\n",
      "Step: 40460 loss: 0.22078\n",
      "Step: 40470 loss: 0.105878\n",
      "Step: 40480 loss: 0.448763\n",
      "Step: 40490 loss: 0.0546038\n",
      "Step: 40500 loss: 0.0985433\n",
      "Step: 40510 loss: 0.368517\n",
      "Step: 40520 loss: 0.0574551\n",
      "Step: 40530 loss: 0.0759502\n",
      "Step: 40540 loss: 0.0394774\n",
      "Step: 40550 loss: 0.449686\n",
      "Step: 40560 loss: 0.159549\n",
      "Step: 40570 loss: 0.468088\n",
      "Step: 40580 loss: 0.248648\n",
      "Step: 40590 loss: 0.584826\n",
      "Step: 40600 loss: 0.381682\n",
      "Step: 40610 loss: 0.0880692\n",
      "Step: 40620 loss: 0.384849\n",
      "Step: 40630 loss: 0.0668172\n",
      "Step: 40640 loss: 0.154968\n",
      "Step: 40650 loss: 0.0699249\n",
      "Step: 40660 loss: 0.0921299\n",
      "Step: 40670 loss: 0.132559\n",
      "Step: 40680 loss: 0.0793482\n",
      "Step: 40690 loss: 0.040044\n",
      "Step: 40700 loss: 0.619115\n",
      "Step: 40710 loss: 0.0489375\n",
      "Step: 40720 loss: 0.045832\n",
      "Step: 40730 loss: 0.0883549\n",
      "Step: 40740 loss: 0.843131\n",
      "Step: 40750 loss: 0.288494\n",
      "Step: 40760 loss: 0.107436\n",
      "Step: 40770 loss: 0.804655\n",
      "Step: 40780 loss: 0.163255\n",
      "Step: 40790 loss: 0.0957087\n",
      "Step: 40800 loss: 0.892067\n",
      "Step: 40810 loss: 0.174594\n",
      "Step: 40820 loss: 0.74134\n",
      "Step: 40830 loss: 0.358748\n",
      "Step: 40840 loss: 0.306498\n",
      "Step: 40850 loss: 0.302739\n",
      "Step: 40860 loss: 0.101532\n",
      "Step: 40870 loss: 0.470931\n",
      "Step: 40880 loss: 0.136573\n",
      "Step: 40890 loss: 0.41407\n",
      "Step: 40900 loss: 0.365364\n",
      "Step: 40910 loss: 0.229175\n",
      "Step: 40920 loss: 0.377972\n",
      "Step: 40930 loss: 0.732484\n",
      "Step: 40940 loss: 0.0415665\n",
      "Step: 40950 loss: 0.174316\n",
      "Step: 40960 loss: 0.579383\n",
      "Step: 40970 loss: 0.210864\n",
      "Step: 40980 loss: 0.698598\n",
      "Step: 40990 loss: 0.293555\n",
      "Step: 41000 loss: 0.189476\n",
      "Saved Model\n",
      "Step: 41010 loss: 0.054692\n",
      "Step: 41020 loss: 0.565787\n",
      "Step: 41030 loss: 0.46793\n",
      "Step: 41040 loss: 0.35505\n",
      "Step: 41050 loss: 0.32837\n",
      "Step: 41060 loss: 0.0596714\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 41070 loss: 0.173165\n",
      "Step: 41080 loss: 0.584465\n",
      "Step: 41090 loss: 0.280949\n",
      "Step: 41100 loss: 0.0586651\n",
      "Step: 41110 loss: 0.0715683\n",
      "Step: 41120 loss: 0.065528\n",
      "Step: 41130 loss: 0.902982\n",
      "Step: 41140 loss: 0.259082\n",
      "Step: 41150 loss: 0.642926\n",
      "Step: 41160 loss: 0.157603\n",
      "Step: 41170 loss: 0.215129\n",
      "Step: 41180 loss: 0.429703\n",
      "Step: 41190 loss: 0.0657954\n",
      "Step: 41200 loss: 0.390801\n",
      "Step: 41210 loss: 0.153557\n",
      "Step: 41220 loss: 0.23228\n",
      "Step: 41230 loss: 0.0738138\n",
      "Step: 41240 loss: 0.071583\n",
      "Step: 41250 loss: 0.0857569\n",
      "Step: 41260 loss: 0.129131\n",
      "Step: 41270 loss: 0.27342\n",
      "Step: 41280 loss: 0.232289\n",
      "Step: 41290 loss: 0.0739835\n",
      "Step: 41300 loss: 0.173283\n",
      "Step: 41310 loss: 0.347164\n",
      "Step: 41320 loss: 0.0608985\n",
      "Step: 41330 loss: 0.258351\n",
      "Step: 41340 loss: 0.264683\n",
      "Step: 41350 loss: 0.554065\n",
      "Step: 41360 loss: 0.124283\n",
      "Step: 41370 loss: 0.480248\n",
      "Step: 41380 loss: 0.130078\n",
      "Step: 41390 loss: 0.479999\n",
      "Step: 41400 loss: 0.12899\n",
      "Step: 41410 loss: 0.0416332\n",
      "Step: 41420 loss: 0.137996\n",
      "Step: 41430 loss: 0.0929829\n",
      "Step: 41440 loss: 0.31998\n",
      "Step: 41450 loss: 0.098417\n",
      "Step: 41460 loss: 0.0475848\n",
      "Step: 41470 loss: 0.0374191\n",
      "Step: 41480 loss: 0.21375\n",
      "Step: 41490 loss: 0.155938\n",
      "Step: 41500 loss: 0.64865\n",
      "Step: 41510 loss: 0.0979094\n",
      "Step: 41520 loss: 0.0611742\n",
      "Step: 41530 loss: 0.87678\n",
      "Step: 41540 loss: 1.1172\n",
      "Step: 41550 loss: 0.171054\n",
      "Step: 41560 loss: 0.292299\n",
      "Step: 41570 loss: 0.495507\n",
      "Step: 41580 loss: 0.533443\n",
      "Step: 41590 loss: 0.242882\n",
      "Step: 41600 loss: 0.167042\n",
      "Step: 41610 loss: 0.135299\n",
      "Step: 41620 loss: 0.630445\n",
      "Step: 41630 loss: 0.250887\n",
      "Step: 41640 loss: 0.769394\n",
      "Step: 41650 loss: 0.0905208\n",
      "Step: 41660 loss: 0.341111\n",
      "Step: 41670 loss: 0.0357731\n",
      "Step: 41680 loss: 0.0682879\n",
      "Step: 41690 loss: 0.0697467\n",
      "Step: 41700 loss: 0.20587\n",
      "Step: 41710 loss: 0.0967957\n",
      "Step: 41720 loss: 1.01157\n",
      "Step: 41730 loss: 0.0912868\n",
      "Step: 41740 loss: 0.111971\n",
      "Step: 41750 loss: 0.141639\n",
      "Step: 41760 loss: 0.352397\n",
      "Step: 41770 loss: 0.1741\n",
      "Step: 41780 loss: 0.793614\n",
      "Step: 41790 loss: 0.214709\n",
      "Step: 41800 loss: 0.606583\n",
      "Step: 41810 loss: 0.0744485\n",
      "Step: 41820 loss: 0.09011\n",
      "Step: 41830 loss: 0.330426\n",
      "Step: 41840 loss: 0.300841\n",
      "Step: 41850 loss: 1.11238\n",
      "Step: 41860 loss: 0.143872\n",
      "Step: 41870 loss: 0.283562\n",
      "Step: 41880 loss: 0.289416\n",
      "Step: 41890 loss: 0.103649\n",
      "Step: 41900 loss: 0.049983\n",
      "Step: 41910 loss: 0.0837227\n",
      "Step: 41920 loss: 0.165176\n",
      "Step: 41930 loss: 0.0448234\n",
      "Step: 41940 loss: 0.0815087\n",
      "Step: 41950 loss: 0.876894\n",
      "Step: 41960 loss: 0.107253\n",
      "Step: 41970 loss: 0.51174\n",
      "Step: 41980 loss: 0.228992\n",
      "Step: 41990 loss: 0.0472922\n",
      "Step: 42000 loss: 0.170452\n",
      "Saved Model\n",
      "Step: 42010 loss: 0.141578\n",
      "Step: 42020 loss: 0.303519\n",
      "Step: 42030 loss: 0.311668\n",
      "Step: 42040 loss: 0.116647\n",
      "Step: 42050 loss: 0.0438727\n",
      "Step: 42060 loss: 0.519154\n",
      "Step: 42070 loss: 0.33702\n",
      "Step: 42080 loss: 0.165364\n",
      "Step: 42090 loss: 0.141005\n",
      "Step: 42100 loss: 0.24868\n",
      "Step: 42110 loss: 0.0791791\n",
      "Step: 42120 loss: 0.102639\n",
      "Step: 42130 loss: 0.270644\n",
      "Step: 42140 loss: 0.260645\n",
      "Step: 42150 loss: 0.638974\n",
      "Step: 42160 loss: 0.0967982\n",
      "Step: 42170 loss: 0.261769\n",
      "Step: 42180 loss: 0.0508329\n",
      "Step: 42190 loss: 0.33576\n",
      "Step: 42200 loss: 0.386328\n",
      "Step: 42210 loss: 0.698448\n",
      "Step: 42220 loss: 0.0748777\n",
      "Step: 42230 loss: 0.134676\n",
      "Step: 42240 loss: 0.656884\n",
      "Step: 42250 loss: 0.0354854\n",
      "Step: 42260 loss: 0.175364\n",
      "Step: 42270 loss: 0.203669\n",
      "Step: 42280 loss: 0.272341\n",
      "Step: 42290 loss: 0.264666\n",
      "Step: 42300 loss: 0.0337302\n",
      "Step: 42310 loss: 0.181381\n",
      "Step: 42320 loss: 0.0902996\n",
      "Step: 42330 loss: 1.30885\n",
      "Step: 42340 loss: 0.288585\n",
      "Step: 42350 loss: 0.20398\n",
      "Step: 42360 loss: 0.20555\n",
      "Step: 42370 loss: 0.619033\n",
      "Step: 42380 loss: 0.345793\n",
      "Step: 42390 loss: 0.147807\n",
      "Step: 42400 loss: 0.461918\n",
      "Step: 42410 loss: 0.504502\n",
      "Step: 42420 loss: 0.745985\n",
      "Step: 42430 loss: 0.124952\n",
      "Step: 42440 loss: 0.112893\n",
      "Step: 42450 loss: 0.114413\n",
      "Step: 42460 loss: 0.319213\n",
      "Step: 42470 loss: 0.220698\n",
      "Step: 42480 loss: 0.891686\n",
      "Step: 42490 loss: 0.119067\n",
      "Step: 42500 loss: 0.235737\n",
      "Step: 42510 loss: 0.0828549\n",
      "Step: 42520 loss: 0.32589\n",
      "Step: 42530 loss: 0.0813653\n",
      "Step: 42540 loss: 0.0420091\n",
      "Step: 42550 loss: 0.092999\n",
      "Step: 42560 loss: 0.577987\n",
      "Step: 42570 loss: 0.177571\n",
      "Step: 42580 loss: 1.12265\n",
      "Step: 42590 loss: 0.935588\n",
      "Step: 42600 loss: 0.0475544\n",
      "Step: 42610 loss: 0.0372456\n",
      "Step: 42620 loss: 0.0408618\n",
      "Step: 42630 loss: 0.336333\n",
      "Step: 42640 loss: 0.279254\n",
      "Step: 42650 loss: 0.16945\n",
      "Step: 42660 loss: 0.193866\n",
      "Step: 42670 loss: 0.0547738\n",
      "Step: 42680 loss: 0.261234\n",
      "Step: 42690 loss: 0.346722\n",
      "Step: 42700 loss: 0.120766\n",
      "Step: 42710 loss: 0.0680116\n",
      "Step: 42720 loss: 0.478971\n",
      "Step: 42730 loss: 0.102522\n",
      "Step: 42740 loss: 0.430954\n",
      "Step: 42750 loss: 0.634644\n",
      "Step: 42760 loss: 0.0390782\n",
      "Step: 42770 loss: 0.118507\n",
      "Step: 42780 loss: 0.176764\n",
      "Step: 42790 loss: 0.685567\n",
      "Step: 42800 loss: 0.332346\n",
      "Step: 42810 loss: 0.932134\n",
      "Step: 42820 loss: 0.232001\n",
      "Step: 42830 loss: 0.208139\n",
      "Step: 42840 loss: 0.0696217\n",
      "Step: 42850 loss: 0.12071\n",
      "Step: 42860 loss: 0.0670347\n",
      "Step: 42870 loss: 0.11665\n",
      "Step: 42880 loss: 0.724104\n",
      "Step: 42890 loss: 0.183754\n",
      "Step: 42900 loss: 0.582062\n",
      "Step: 42910 loss: 0.501547\n",
      "Step: 42920 loss: 0.197123\n",
      "Step: 42930 loss: 0.0877379\n",
      "Step: 42940 loss: 0.235848\n",
      "Step: 42950 loss: 0.0881113\n",
      "Step: 42960 loss: 0.0714766\n",
      "Step: 42970 loss: 0.467168\n",
      "Step: 42980 loss: 0.126741\n",
      "Step: 42990 loss: 0.570034\n",
      "Step: 43000 loss: 0.096325\n",
      "Saved Model\n",
      "Step: 43010 loss: 0.0415312\n",
      "Step: 43020 loss: 0.0502137\n",
      "Step: 43030 loss: 0.254877\n",
      "Step: 43040 loss: 0.072514\n",
      "Step: 43050 loss: 0.104332\n",
      "Step: 43060 loss: 0.0924486\n",
      "Step: 43070 loss: 0.270854\n",
      "Step: 43080 loss: 0.143398\n",
      "Step: 43090 loss: 0.352022\n",
      "Step: 43100 loss: 0.544763\n",
      "Step: 43110 loss: 0.489986\n",
      "Step: 43120 loss: 0.0900348\n",
      "Step: 43130 loss: 1.6811\n",
      "Step: 43140 loss: 0.501099\n",
      "Step: 43150 loss: 0.169574\n",
      "Step: 43160 loss: 0.0561932\n",
      "Step: 43170 loss: 0.130611\n",
      "Step: 43180 loss: 0.553631\n",
      "Step: 43190 loss: 0.304437\n",
      "Step: 43200 loss: 0.119676\n",
      "Step: 43210 loss: 0.036138\n",
      "Step: 43220 loss: 0.751895\n",
      "Step: 43230 loss: 0.33646\n",
      "Step: 43240 loss: 0.348454\n",
      "Step: 43250 loss: 0.539578\n",
      "Step: 43260 loss: 0.118776\n",
      "Step: 43270 loss: 0.0526686\n",
      "Step: 43280 loss: 0.0379632\n",
      "Step: 43290 loss: 0.190601\n",
      "Step: 43300 loss: 0.31924\n",
      "Step: 43310 loss: 0.187523\n",
      "Step: 43320 loss: 0.404171\n",
      "Step: 43330 loss: 0.385106\n",
      "Step: 43340 loss: 0.0831939\n",
      "Step: 43350 loss: 0.213377\n",
      "Step: 43360 loss: 0.715135\n",
      "Step: 43370 loss: 0.122558\n",
      "Step: 43380 loss: 0.877118\n",
      "Step: 43390 loss: 0.324347\n",
      "Step: 43400 loss: 0.0616385\n",
      "Step: 43410 loss: 0.0569564\n",
      "Step: 43420 loss: 0.240412\n",
      "Step: 43430 loss: 0.183496\n",
      "Step: 43440 loss: 0.0603827\n",
      "Step: 43450 loss: 0.579877\n",
      "Step: 43460 loss: 0.180894\n",
      "Step: 43470 loss: 0.770099\n",
      "Step: 43480 loss: 0.0947162\n",
      "Step: 43490 loss: 0.597037\n",
      "Step: 43500 loss: 0.122569\n",
      "Step: 43510 loss: 0.102572\n",
      "Step: 43520 loss: 0.362568\n",
      "Step: 43530 loss: 0.228913\n",
      "Step: 43540 loss: 0.259715\n",
      "Step: 43550 loss: 0.478537\n",
      "Step: 43560 loss: 0.0638408\n",
      "Step: 43570 loss: 0.0513115\n",
      "Step: 43580 loss: 0.56234\n",
      "Step: 43590 loss: 0.32212\n",
      "Step: 43600 loss: 0.0545326\n",
      "Step: 43610 loss: 0.233905\n",
      "Step: 43620 loss: 0.151591\n",
      "Step: 43630 loss: 0.0375235\n",
      "Step: 43640 loss: 0.423939\n",
      "Step: 43650 loss: 0.84154\n",
      "Step: 43660 loss: 0.26316\n",
      "Step: 43670 loss: 0.435576\n",
      "Step: 43680 loss: 0.123589\n",
      "Step: 43690 loss: 0.196496\n",
      "Step: 43700 loss: 0.331946\n",
      "Step: 43710 loss: 0.589067\n",
      "Step: 43720 loss: 0.576039\n",
      "Step: 43730 loss: 0.626806\n",
      "Step: 43740 loss: 0.328856\n",
      "Step: 43750 loss: 0.101093\n",
      "Step: 43760 loss: 0.160884\n",
      "Step: 43770 loss: 0.0636871\n",
      "Step: 43780 loss: 0.0605466\n",
      "Step: 43790 loss: 0.333031\n",
      "Step: 43800 loss: 0.135636\n",
      "Step: 43810 loss: 0.191077\n",
      "Step: 43820 loss: 0.0662777\n",
      "Step: 43830 loss: 0.510305\n",
      "Step: 43840 loss: 0.0813577\n",
      "Step: 43850 loss: 0.0391253\n",
      "Step: 43860 loss: 0.0456333\n",
      "Step: 43870 loss: 0.214085\n",
      "Step: 43880 loss: 0.0461376\n",
      "Step: 43890 loss: 0.0470099\n",
      "Step: 43900 loss: 0.0792278\n",
      "Step: 43910 loss: 0.448328\n",
      "Step: 43920 loss: 0.0333508\n",
      "Step: 43930 loss: 0.590826\n",
      "Step: 43940 loss: 0.577505\n",
      "Step: 43950 loss: 0.0911679\n",
      "Step: 43960 loss: 0.395338\n",
      "Step: 43970 loss: 0.303869\n",
      "Step: 43980 loss: 0.426541\n",
      "Step: 43990 loss: 0.272663\n",
      "Step: 44000 loss: 0.820847\n",
      "Saved Model\n",
      "Step: 44010 loss: 0.102388\n",
      "Step: 44020 loss: 0.708378\n",
      "Step: 44030 loss: 0.637476\n",
      "Step: 44040 loss: 0.866369\n",
      "Step: 44050 loss: 0.136692\n",
      "Step: 44060 loss: 0.0342482\n",
      "Step: 44070 loss: 0.0739084\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 44080 loss: 0.128725\n",
      "Step: 44090 loss: 0.300282\n",
      "Step: 44100 loss: 0.0374196\n",
      "Step: 44110 loss: 0.148153\n",
      "Step: 44120 loss: 0.910608\n",
      "Step: 44130 loss: 0.0519017\n",
      "Step: 44140 loss: 0.0320734\n",
      "Step: 44150 loss: 0.0778119\n",
      "Step: 44160 loss: 0.884712\n",
      "Step: 44170 loss: 0.0930497\n",
      "Step: 44180 loss: 0.98539\n",
      "Step: 44190 loss: 0.0657582\n",
      "Step: 44200 loss: 0.0381521\n",
      "Step: 44210 loss: 0.204958\n",
      "Step: 44220 loss: 0.126904\n",
      "Step: 44230 loss: 0.361081\n",
      "Step: 44240 loss: 0.126165\n",
      "Step: 44250 loss: 0.204496\n",
      "Step: 44260 loss: 0.197386\n",
      "Step: 44270 loss: 0.0427434\n",
      "Step: 44280 loss: 0.102043\n",
      "Step: 44290 loss: 0.79481\n",
      "Step: 44300 loss: 0.0462335\n",
      "Step: 44310 loss: 0.0454056\n",
      "Step: 44320 loss: 0.160095\n",
      "Step: 44330 loss: 0.208157\n",
      "Step: 44340 loss: 0.126203\n",
      "Step: 44350 loss: 0.436138\n",
      "Step: 44360 loss: 0.323827\n",
      "Step: 44370 loss: 0.379715\n",
      "Step: 44380 loss: 0.210826\n",
      "Step: 44390 loss: 0.200252\n",
      "Step: 44400 loss: 0.672188\n",
      "Step: 44410 loss: 0.0782018\n",
      "Step: 44420 loss: 0.34085\n",
      "Step: 44430 loss: 0.0445239\n",
      "Step: 44440 loss: 0.0340189\n",
      "Step: 44450 loss: 0.0436254\n",
      "Step: 44460 loss: 0.14269\n",
      "Step: 44470 loss: 0.31765\n",
      "Step: 44480 loss: 0.393852\n",
      "Step: 44490 loss: 0.106653\n",
      "Step: 44500 loss: 0.133938\n",
      "Step: 44510 loss: 0.462677\n",
      "Step: 44520 loss: 0.142229\n",
      "Step: 44530 loss: 0.247611\n",
      "Step: 44540 loss: 0.44417\n",
      "Step: 44550 loss: 0.28131\n",
      "Step: 44560 loss: 0.257264\n",
      "Step: 44570 loss: 1.20045\n",
      "Step: 44580 loss: 0.134708\n",
      "Step: 44590 loss: 0.737727\n",
      "Step: 44600 loss: 0.486038\n",
      "Step: 44610 loss: 0.108811\n",
      "Step: 44620 loss: 0.958808\n",
      "Step: 44630 loss: 0.415297\n",
      "Step: 44640 loss: 0.15889\n",
      "Step: 44650 loss: 0.195348\n",
      "Step: 44660 loss: 0.0401683\n",
      "Step: 44670 loss: 0.21581\n",
      "Step: 44680 loss: 0.987038\n",
      "Step: 44690 loss: 0.0512616\n",
      "Step: 44700 loss: 0.0826724\n",
      "Step: 44710 loss: 0.550879\n",
      "Step: 44720 loss: 0.045283\n",
      "Step: 44730 loss: 0.47679\n",
      "Step: 44740 loss: 0.381925\n",
      "Step: 44750 loss: 0.367138\n",
      "Step: 44760 loss: 0.0717046\n",
      "Step: 44770 loss: 1.09094\n",
      "Step: 44780 loss: 0.386318\n",
      "Step: 44790 loss: 0.0804961\n",
      "Step: 44800 loss: 0.735789\n",
      "Step: 44810 loss: 0.0728073\n",
      "Step: 44820 loss: 1.03278\n",
      "Step: 44830 loss: 0.277967\n",
      "Step: 44840 loss: 0.976018\n",
      "Step: 44850 loss: 0.901406\n",
      "Step: 44860 loss: 0.41772\n",
      "Step: 44870 loss: 0.0419873\n",
      "Step: 44880 loss: 0.10083\n",
      "Step: 44890 loss: 0.51115\n",
      "Step: 44900 loss: 0.0396068\n",
      "Step: 44910 loss: 0.302241\n",
      "Step: 44920 loss: 0.4483\n",
      "Step: 44930 loss: 0.292268\n",
      "Step: 44940 loss: 0.0337343\n",
      "Step: 44950 loss: 0.0736216\n",
      "Step: 44960 loss: 0.303457\n",
      "Step: 44970 loss: 0.073259\n",
      "Step: 44980 loss: 0.333311\n",
      "Step: 44990 loss: 0.115186\n",
      "Step: 45000 loss: 0.142253\n",
      "Saved Model\n",
      "Step: 45010 loss: 0.0466814\n",
      "Step: 45020 loss: 0.107431\n",
      "Step: 45030 loss: 0.374277\n",
      "Step: 45040 loss: 0.190482\n",
      "Step: 45050 loss: 0.094342\n",
      "Step: 45060 loss: 0.439292\n",
      "Step: 45070 loss: 0.110958\n",
      "Step: 45080 loss: 0.198202\n",
      "Step: 45090 loss: 0.0875918\n",
      "Step: 45100 loss: 0.0381937\n",
      "Step: 45110 loss: 0.148952\n",
      "Step: 45120 loss: 0.11817\n",
      "Step: 45130 loss: 0.538672\n",
      "Step: 45140 loss: 0.0908321\n",
      "Step: 45150 loss: 0.696084\n",
      "Step: 45160 loss: 0.0777248\n",
      "Step: 45170 loss: 0.196694\n",
      "Step: 45180 loss: 0.162114\n",
      "Step: 45190 loss: 0.167444\n",
      "Step: 45200 loss: 0.093216\n",
      "Step: 45210 loss: 0.686359\n",
      "Step: 45220 loss: 0.0818628\n",
      "Step: 45230 loss: 0.116829\n",
      "Step: 45240 loss: 0.0436674\n",
      "Step: 45250 loss: 0.0717334\n",
      "Step: 45260 loss: 0.0360142\n",
      "Step: 45270 loss: 0.0776025\n",
      "Step: 45280 loss: 0.169896\n",
      "Step: 45290 loss: 0.113072\n",
      "Step: 45300 loss: 0.303172\n",
      "Step: 45310 loss: 0.654293\n",
      "Step: 45320 loss: 0.133675\n",
      "Step: 45330 loss: 0.155818\n",
      "Step: 45340 loss: 0.1791\n",
      "Step: 45350 loss: 0.269838\n",
      "Step: 45360 loss: 0.368444\n",
      "Step: 45370 loss: 0.295538\n",
      "Step: 45380 loss: 0.189683\n",
      "Step: 45390 loss: 0.754578\n",
      "Step: 45400 loss: 0.279071\n",
      "Step: 45410 loss: 0.044742\n",
      "Step: 45420 loss: 0.0412033\n",
      "Step: 45430 loss: 0.443697\n",
      "Step: 45440 loss: 0.381422\n",
      "Step: 45450 loss: 0.0469469\n",
      "Step: 45460 loss: 0.153267\n",
      "Step: 45470 loss: 0.451778\n",
      "Step: 45480 loss: 0.0601029\n",
      "Step: 45490 loss: 0.298515\n",
      "Step: 45500 loss: 0.051668\n",
      "Step: 45510 loss: 0.950147\n",
      "Step: 45520 loss: 0.039203\n",
      "Step: 45530 loss: 0.449392\n",
      "Step: 45540 loss: 0.837219\n",
      "Step: 45550 loss: 0.176624\n",
      "Step: 45560 loss: 0.379374\n",
      "Step: 45570 loss: 0.1504\n",
      "Step: 45580 loss: 0.653903\n",
      "Step: 45590 loss: 0.194013\n",
      "Step: 45600 loss: 0.417334\n",
      "Step: 45610 loss: 0.262239\n",
      "Step: 45620 loss: 0.25404\n",
      "Step: 45630 loss: 0.107939\n",
      "Step: 45640 loss: 0.071364\n",
      "Step: 45650 loss: 0.253467\n",
      "Step: 45660 loss: 0.0510766\n",
      "Step: 45670 loss: 0.080856\n",
      "Step: 45680 loss: 0.266987\n",
      "Step: 45690 loss: 0.156394\n",
      "Step: 45700 loss: 0.758438\n",
      "Step: 45710 loss: 0.0789872\n",
      "Step: 45720 loss: 0.449504\n",
      "Step: 45730 loss: 0.0530383\n",
      "Step: 45740 loss: 0.228946\n",
      "Step: 45750 loss: 0.127175\n",
      "Step: 45760 loss: 0.360998\n",
      "Step: 45770 loss: 0.0554608\n",
      "Step: 45780 loss: 0.586594\n",
      "Step: 45790 loss: 0.268456\n",
      "Step: 45800 loss: 0.244963\n",
      "Step: 45810 loss: 0.139351\n",
      "Step: 45820 loss: 0.187211\n",
      "Step: 45830 loss: 0.111472\n",
      "Step: 45840 loss: 0.200015\n",
      "Step: 45850 loss: 0.0920665\n",
      "Step: 45860 loss: 0.272127\n",
      "Step: 45870 loss: 0.117205\n",
      "Step: 45880 loss: 0.0963999\n",
      "Step: 45890 loss: 0.0508202\n",
      "Step: 45900 loss: 0.0777427\n",
      "Step: 45910 loss: 0.270353\n",
      "Step: 45920 loss: 0.437204\n",
      "Step: 45930 loss: 0.208142\n",
      "Step: 45940 loss: 0.318623\n",
      "Step: 45950 loss: 0.266413\n",
      "Step: 45960 loss: 0.0391937\n",
      "Step: 45970 loss: 0.061859\n",
      "Step: 45980 loss: 0.0852186\n",
      "Step: 45990 loss: 0.382121\n",
      "Step: 46000 loss: 0.22757\n",
      "Saved Model\n",
      "Step: 46010 loss: 0.581056\n",
      "Step: 46020 loss: 0.201717\n",
      "Step: 46030 loss: 0.36308\n",
      "Step: 46040 loss: 0.0860861\n",
      "Step: 46050 loss: 0.118153\n",
      "Step: 46060 loss: 0.756183\n",
      "Step: 46070 loss: 0.43011\n",
      "Step: 46080 loss: 0.0592258\n",
      "Step: 46090 loss: 0.121594\n",
      "Step: 46100 loss: 0.155105\n",
      "Step: 46110 loss: 0.174857\n",
      "Step: 46120 loss: 0.135156\n",
      "Step: 46130 loss: 0.18071\n",
      "Step: 46140 loss: 0.278001\n",
      "Step: 46150 loss: 0.113957\n",
      "Step: 46160 loss: 0.486552\n",
      "Step: 46170 loss: 0.300927\n",
      "Step: 46180 loss: 0.187184\n",
      "Step: 46190 loss: 0.573333\n",
      "Step: 46200 loss: 0.46631\n",
      "Step: 46210 loss: 0.0441527\n",
      "Step: 46220 loss: 0.0671836\n",
      "Step: 46230 loss: 0.394186\n",
      "Step: 46240 loss: 0.633353\n",
      "Step: 46250 loss: 0.217326\n",
      "Step: 46260 loss: 0.304994\n",
      "Step: 46270 loss: 0.198606\n",
      "Step: 46280 loss: 0.209601\n",
      "Step: 46290 loss: 0.0416074\n",
      "Step: 46300 loss: 0.0396832\n",
      "Step: 46310 loss: 0.340327\n",
      "Step: 46320 loss: 0.119747\n",
      "Step: 46330 loss: 0.0719539\n",
      "Step: 46340 loss: 0.864477\n",
      "Step: 46350 loss: 0.367887\n",
      "Step: 46360 loss: 0.174352\n",
      "Step: 46370 loss: 0.611849\n",
      "Step: 46380 loss: 0.134003\n",
      "Step: 46390 loss: 0.587879\n",
      "Step: 46400 loss: 0.315072\n",
      "Step: 46410 loss: 0.367741\n",
      "Step: 46420 loss: 0.116978\n",
      "Step: 46430 loss: 0.145544\n",
      "Step: 46440 loss: 0.153234\n",
      "Step: 46450 loss: 0.0962666\n",
      "Step: 46460 loss: 0.127044\n",
      "Step: 46470 loss: 0.226739\n",
      "Step: 46480 loss: 0.0974501\n",
      "Step: 46490 loss: 0.35769\n",
      "Step: 46500 loss: 0.0648174\n",
      "Step: 46510 loss: 0.178324\n",
      "Step: 46520 loss: 0.459085\n",
      "Step: 46530 loss: 0.0576207\n",
      "Step: 46540 loss: 0.034969\n",
      "Step: 46550 loss: 0.0375493\n",
      "Step: 46560 loss: 0.422198\n",
      "Step: 46570 loss: 0.191319\n",
      "Step: 46580 loss: 0.304332\n",
      "Step: 46590 loss: 0.0588432\n",
      "Step: 46600 loss: 0.177074\n",
      "Step: 46610 loss: 0.0398025\n",
      "Step: 46620 loss: 0.0707869\n",
      "Step: 46630 loss: 0.0915346\n",
      "Step: 46640 loss: 0.328638\n",
      "Step: 46650 loss: 0.558428\n",
      "Step: 46660 loss: 0.125269\n",
      "Step: 46670 loss: 0.205949\n",
      "Step: 46680 loss: 0.118795\n",
      "Step: 46690 loss: 0.686084\n",
      "Step: 46700 loss: 0.353131\n",
      "Step: 46710 loss: 0.175936\n",
      "Step: 46720 loss: 0.19873\n",
      "Step: 46730 loss: 0.0842294\n",
      "Step: 46740 loss: 0.101918\n",
      "Step: 46750 loss: 0.251921\n",
      "Step: 46760 loss: 0.324678\n",
      "Step: 46770 loss: 0.12477\n",
      "Step: 46780 loss: 0.0504275\n",
      "Step: 46790 loss: 0.874196\n",
      "Step: 46800 loss: 0.701008\n",
      "Step: 46810 loss: 0.4881\n",
      "Step: 46820 loss: 0.0810837\n",
      "Step: 46830 loss: 0.0590493\n",
      "Step: 46840 loss: 0.0811057\n",
      "Step: 46850 loss: 0.041064\n",
      "Step: 46860 loss: 0.0394733\n",
      "Step: 46870 loss: 0.491598\n",
      "Step: 46880 loss: 0.231077\n",
      "Step: 46890 loss: 0.692792\n",
      "Step: 46900 loss: 0.335875\n",
      "Step: 46910 loss: 0.0645471\n",
      "Step: 46920 loss: 0.165322\n",
      "Step: 46930 loss: 0.259845\n",
      "Step: 46940 loss: 0.052609\n",
      "Step: 46950 loss: 0.138301\n",
      "Step: 46960 loss: 0.118634\n",
      "Step: 46970 loss: 0.407385\n",
      "Step: 46980 loss: 0.126699\n",
      "Step: 46990 loss: 0.0584954\n",
      "Step: 47000 loss: 0.454632\n",
      "Saved Model\n",
      "Step: 47010 loss: 0.0649546\n",
      "Step: 47020 loss: 0.0439936\n",
      "Step: 47030 loss: 0.0353501\n",
      "Step: 47040 loss: 0.290381\n",
      "Step: 47050 loss: 0.0709093\n",
      "Step: 47060 loss: 0.0403258\n",
      "Step: 47070 loss: 0.414105\n",
      "Step: 47080 loss: 0.0832375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 47090 loss: 0.48607\n",
      "Step: 47100 loss: 0.0445481\n",
      "Step: 47110 loss: 0.107045\n",
      "Step: 47120 loss: 0.157856\n",
      "Step: 47130 loss: 0.689522\n",
      "Step: 47140 loss: 0.356236\n",
      "Step: 47150 loss: 0.0575687\n",
      "Step: 47160 loss: 0.880571\n",
      "Step: 47170 loss: 0.225042\n",
      "Step: 47180 loss: 1.22775\n",
      "Step: 47190 loss: 0.126019\n",
      "Step: 47200 loss: 0.45015\n",
      "Step: 47210 loss: 0.175434\n",
      "Step: 47220 loss: 0.341063\n",
      "Step: 47230 loss: 0.623115\n",
      "Step: 47240 loss: 0.909092\n",
      "Step: 47250 loss: 0.066878\n",
      "Step: 47260 loss: 0.11269\n",
      "Step: 47270 loss: 0.162067\n",
      "Step: 47280 loss: 0.0739465\n",
      "Step: 47290 loss: 0.214058\n",
      "Step: 47300 loss: 0.165262\n",
      "Step: 47310 loss: 0.0769133\n",
      "Step: 47320 loss: 0.740466\n",
      "Step: 47330 loss: 0.0842959\n",
      "Step: 47340 loss: 0.0396902\n",
      "Step: 47350 loss: 0.14395\n",
      "Step: 47360 loss: 0.0829715\n",
      "Step: 47370 loss: 0.0411171\n",
      "Step: 47380 loss: 0.691197\n",
      "Step: 47390 loss: 0.451377\n",
      "Step: 47400 loss: 0.19287\n",
      "Step: 47410 loss: 0.0343534\n",
      "Step: 47420 loss: 0.19025\n",
      "Step: 47430 loss: 0.452724\n",
      "Step: 47440 loss: 0.0943098\n",
      "Step: 47450 loss: 0.386456\n",
      "Step: 47460 loss: 0.368244\n",
      "Step: 47470 loss: 0.118861\n",
      "Step: 47480 loss: 0.120759\n",
      "Step: 47490 loss: 0.206076\n",
      "Step: 47500 loss: 0.0367699\n",
      "Step: 47510 loss: 0.416655\n",
      "Step: 47520 loss: 0.31104\n",
      "Step: 47530 loss: 0.0354947\n",
      "Step: 47540 loss: 0.237446\n",
      "Step: 47550 loss: 0.0745112\n",
      "Step: 47560 loss: 0.0419025\n",
      "Step: 47570 loss: 0.53912\n",
      "Step: 47580 loss: 0.264892\n",
      "Step: 47590 loss: 0.890957\n",
      "Step: 47600 loss: 0.103646\n",
      "Step: 47610 loss: 0.0475121\n",
      "Step: 47620 loss: 0.127736\n",
      "Step: 47630 loss: 0.162011\n",
      "Step: 47640 loss: 0.155866\n",
      "Step: 47650 loss: 0.130471\n",
      "Step: 47660 loss: 0.0385213\n",
      "Step: 47670 loss: 0.167263\n",
      "Step: 47680 loss: 0.227582\n",
      "Step: 47690 loss: 0.121266\n",
      "Step: 47700 loss: 0.58036\n",
      "Step: 47710 loss: 0.512977\n",
      "Step: 47720 loss: 0.0604548\n",
      "Step: 47730 loss: 0.165498\n",
      "Step: 47740 loss: 0.277407\n",
      "Step: 47750 loss: 0.234889\n",
      "Step: 47760 loss: 0.509203\n",
      "Step: 47770 loss: 0.160462\n",
      "Step: 47780 loss: 0.12678\n",
      "Step: 47790 loss: 0.344552\n",
      "Step: 47800 loss: 0.328056\n",
      "Step: 47810 loss: 0.121361\n",
      "Step: 47820 loss: 0.183876\n",
      "Step: 47830 loss: 0.226187\n",
      "Step: 47840 loss: 0.393935\n",
      "Step: 47850 loss: 0.209987\n",
      "Step: 47860 loss: 0.0542167\n",
      "Step: 47870 loss: 0.5935\n",
      "Step: 47880 loss: 0.18431\n",
      "Step: 47890 loss: 0.26464\n",
      "Step: 47900 loss: 0.0478429\n",
      "Step: 47910 loss: 0.282553\n",
      "Step: 47920 loss: 0.491961\n",
      "Step: 47930 loss: 0.579593\n",
      "Step: 47940 loss: 0.536961\n",
      "Step: 47950 loss: 0.195642\n",
      "Step: 47960 loss: 0.0526661\n",
      "Step: 47970 loss: 0.234355\n",
      "Step: 47980 loss: 0.179904\n",
      "Step: 47990 loss: 0.106843\n",
      "Step: 48000 loss: 0.526727\n",
      "Saved Model\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 20\n",
    "for i in range(num_epochs):\n",
    "    for j in range(num_batches):\n",
    "        path_dict = [path1_len[j*batch_size:(j+1)*batch_size], path2_len[j*batch_size:(j+1)*batch_size]]\n",
    "        word_dict = [word_p1_ids[j*batch_size:(j+1)*batch_size], word_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "        pos_dict = [pos_p1_ids[j*batch_size:(j+1)*batch_size], pos_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "        dep_dict = [dep_p1_ids[j*batch_size:(j+1)*batch_size], dep_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "        y_dict = rel_ids[j*batch_size:(j+1)*batch_size]\n",
    "        \n",
    "        feed_dict = {\n",
    "            path_length:path_dict,\n",
    "            word_ids:word_dict,\n",
    "            pos_ids:pos_dict,\n",
    "            dep_ids:dep_dict,\n",
    "            y:y_dict}\n",
    "        _, loss, step = sess.run([optimizer, total_loss, global_step], feed_dict)\n",
    "        if step%10==0:\n",
    "            print(\"Step:\", step, \"loss:\",loss)\n",
    "        if step % 1000 == 0:\n",
    "            saver.save(sess, model_dir + '/model')\n",
    "            print(\"Saved Model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy 92.2125\n"
     ]
    }
   ],
   "source": [
    "# training accuracy\n",
    "all_predictions = []\n",
    "for j in range(num_batches):\n",
    "    path_dict = [path1_len[j*batch_size:(j+1)*batch_size], path2_len[j*batch_size:(j+1)*batch_size]]\n",
    "    word_dict = [word_p1_ids[j*batch_size:(j+1)*batch_size], word_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    pos_dict = [pos_p1_ids[j*batch_size:(j+1)*batch_size], pos_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    dep_dict = [dep_p1_ids[j*batch_size:(j+1)*batch_size], dep_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    y_dict = rel_ids[j*batch_size:(j+1)*batch_size]\n",
    "\n",
    "    feed_dict = {\n",
    "        path_length:path_dict,\n",
    "        word_ids:word_dict,\n",
    "        pos_ids:pos_dict,\n",
    "        dep_ids:dep_dict,\n",
    "        y:y_dict}\n",
    "    batch_predictions = sess.run(predictions, feed_dict)\n",
    "    all_predictions.append(batch_predictions)\n",
    "\n",
    "y_pred = []\n",
    "for i in range(num_batches):\n",
    "    for pred in all_predictions[i]:\n",
    "        y_pred.append(pred)\n",
    "\n",
    "count = 0\n",
    "for i in range(batch_size*num_batches):\n",
    "    count += y_pred[i]==rel_ids[i]\n",
    "accuracy = count/(batch_size*num_batches) * 100\n",
    "\n",
    "print(\"training accuracy\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open(data_dir + '/test_paths', 'rb')\n",
    "word_p1, word_p2, dep_p1, dep_p2, pos_p1, pos_p2 = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "relations = []\n",
    "for line in open(data_dir + '/test_relations.txt'):\n",
    "    relations.append(line.strip().split()[0])\n",
    "\n",
    "length = len(word_p1)\n",
    "num_batches = int(length/batch_size)\n",
    "\n",
    "for i in range(length):\n",
    "    for j, word in enumerate(word_p1[i]):\n",
    "        word = word.lower()\n",
    "        word_p1[i][j] = word if word in word2id else unknown_token \n",
    "    for k, word in enumerate(word_p2[i]):\n",
    "        word = word.lower()\n",
    "        word_p2[i][k] = word if word in word2id else unknown_token \n",
    "    for l, d in enumerate(dep_p1[i]):\n",
    "        dep_p1[i][l] = d if d in dep2id else 'OTH'\n",
    "    for m, d in enumerate(dep_p2[i]):\n",
    "        dep_p2[i][m] = d if d in dep2id else 'OTH'\n",
    "\n",
    "word_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "word_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "pos_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "pos_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "dep_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "dep_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "rel_ids = np.array([rel2id[rel] for rel in relations])\n",
    "path1_len = np.array([len(w) for w in word_p1], dtype=int)\n",
    "path2_len = np.array([len(w) for w in word_p2])\n",
    "\n",
    "for i in range(length):\n",
    "    for j, w in enumerate(word_p1[i]):\n",
    "        word_p1_ids[i][j] = word2id[w]\n",
    "    for j, w in enumerate(word_p2[i]):\n",
    "        word_p2_ids[i][j] = word2id[w]\n",
    "    for j, w in enumerate(pos_p1[i]):\n",
    "        pos_p1_ids[i][j] = pos_tag(w)\n",
    "    for j, w in enumerate(pos_p2[i]):\n",
    "        pos_p2_ids[i][j] = pos_tag(w)\n",
    "    for j, w in enumerate(dep_p1[i]):\n",
    "        dep_p1_ids[i][j] = dep2id[w]\n",
    "    for j, w in enumerate(dep_p2[i]):\n",
    "        dep_p2_ids[i][j] = dep2id[w]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy 62.2878228782\n"
     ]
    }
   ],
   "source": [
    "# test \n",
    "all_predictions = []\n",
    "for j in range(num_batches):\n",
    "    path_dict = [path1_len[j*batch_size:(j+1)*batch_size], path2_len[j*batch_size:(j+1)*batch_size]]\n",
    "    word_dict = [word_p1_ids[j*batch_size:(j+1)*batch_size], word_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    pos_dict = [pos_p1_ids[j*batch_size:(j+1)*batch_size], pos_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    dep_dict = [dep_p1_ids[j*batch_size:(j+1)*batch_size], dep_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    y_dict = rel_ids[j*batch_size:(j+1)*batch_size]\n",
    "\n",
    "    feed_dict = {\n",
    "        path_length:path_dict,\n",
    "        word_ids:word_dict,\n",
    "        pos_ids:pos_dict,\n",
    "        dep_ids:dep_dict,\n",
    "        y:y_dict}\n",
    "    batch_predictions = sess.run(predictions, feed_dict)\n",
    "    all_predictions.append(batch_predictions)\n",
    "\n",
    "y_pred = []\n",
    "for i in range(num_batches):\n",
    "    for pred in all_predictions[i]:\n",
    "        y_pred.append(pred)\n",
    "\n",
    "count = 0\n",
    "for i in range(batch_size*num_batches):\n",
    "    count += y_pred[i]==rel_ids[i]\n",
    "accuracy = count/(batch_size*num_batches) * 100\n",
    "\n",
    "print(\"test accuracy\", accuracy)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
